# coding:utf-8

from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from scrapy_test.items import MusicItem, MusicReviewItem


import re


class ReviewSpider(CrawlSpider):
    name = 'mtl'
    allowed_domains = ['www.meitulu.com']
    start_urls = ['https://www.meitulu.com/item/18000_3.html']
    # 当编写爬虫规则时，请避免使用parse作为回调函数。 由于CrawlSpider使用parse方法来实现其逻辑，
    # 如果您覆盖了parse方法，crawl spider 将会运行失败。 http://www.meizitu.com/a/5476.html
    rules = (
        # Rule(LinkExtractor(allow=r"/subject/\d+/reviews$")),
        # Rule(LinkExtractor(allow=r"/subject/\d+/reviews\?sort=time$")),
        # Rule(LinkExtractor(allow=r"/subject/\d+/reviews\?sort=time\&start=\d+$")),
        Rule(LinkExtractor(allow=r"/a/\d+/.html$"), callback="parse_review", follow=True),
    )

    def parse_review(self, response):
        try:
            item = MusicReviewItem()
            item['review_title'] = "".join(response.xpath('//*[@property="v:summary"]/text()').extract())
            content = "".join(
                response.xpath('//*[@id="link-report"]/div[@property="v:description"]/text()').extract())
            item['review_content'] = content.lstrip().rstrip().replace("\n", " ")
            item['review_author'] = "".join(response.xpath('//*[@property = "v:reviewer"]/text()').extract())
            item['review_music'] = "".join(response.xpath('//*[@class="main-hd"]/a[2]/text()').extract())
            item['review_time'] = "".join(response.xpath('//*[@class="main-hd"]/p/text()').extract())
            item['review_url'] = response.url
            yield item
        except Exception as error:
            self.log(error)